{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1600521976821",
   "display_name": "Python 3.8.5 64-bit ('myenv': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Building prefix dict from the default dictionary ...\nLoading model from cache C:\\Users\\ADMINI~1.DES\\AppData\\Local\\Temp\\jieba.cache\nLoading model cost 0.471 seconds.\nPrefix dict has been built successfully.\n"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import jieba\n",
    "\n",
    "\n",
    "def print_all_file_path(init_file_path, keyword):\n",
    "    paths = []\n",
    "    for cur_dir, sub_dir, included_file in os.walk(init_file_path):\n",
    "        if included_file:\n",
    "            for file in included_file:\n",
    "                if re.search(keyword, file):\n",
    "                    paths.append(cur_dir + \"\\\\\" + file)\n",
    "    return paths\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "####加载第一个文件的内容\n",
    "\n",
    "paths1 = [r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\dev-v2.0.json',r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\train-v2.0.json']\n",
    "\n",
    "allLine =  []\n",
    "for path in paths1:\n",
    "    with open(path,'r',encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            allLine.append(line)\n",
    "\n",
    "allText = []\n",
    "for i in range(len(json.loads(allLine[0]).get('data')[0].get('paragraphs'))):\n",
    "    for j in range(len(json.loads(allLine[0]).get('data')[0].get('paragraphs')[i-1].get('qas'))):\n",
    "        allText.append(json.loads(allLine[0]).get('data')[0].get('paragraphs')[i-1].get('qas')[j-1].get('question'))\n",
    "        allText.append(json.loads(allLine[0]).get('data')[0].get('paragraphs')[i-1].get('context'))\n",
    "        allText.append(json.loads(allLine[0]).get('data')[0].get('paragraphs')[i-1].get('qas')[j-1].get('answers'))\n",
    "        \n",
    "\n",
    "\n",
    "###加载第二个文件的内容\n",
    "# paths\n",
    "path =r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed'\n",
    "paths = print_all_file_path(path, \".json\")\n",
    "\n",
    "allLine =  []\n",
    "for path in paths[0:1]:\n",
    "    with open(path,'r',encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            allLine.append(line)\n",
    "\n",
    "for line in allLine:\n",
    "    if line is not None:\n",
    "        if json.loads(line).get('documents')[0].get('paragraphs') is not None:\n",
    "            allText.append(json.loads(line).get('documents')[0].get('paragraphs'))\n",
    "        if json.loads(line).get('documents')[0].get('segmented_title') is not None:    \n",
    "            allText.append(json.loads(line).get('documents')[0].get('segmented_title'))\n",
    "        if json.loads(line).get('documents')[0].get('segmented_paragraphs') is not None:\n",
    "            allText.append(json.loads(line).get('documents')[0].get('segmented_paragraphs'))\n",
    "        if json.loads(line).get('documents')[0].get('title') is not None:\n",
    "            allText.append(json.loads(line).get('documents')[0].get('title'))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# ######################训练词向量\n",
    "\n",
    "# fileRead = []\n",
    "# for file in newpaths:\n",
    "#     with open(file,'r',encoding='utf-8') as fileTrainRaw:\n",
    "#         for line in fileTrainRaw:\n",
    "#             fileRead.append(line)\n",
    "# print(fileRead)\n",
    "\n",
    "\n",
    "\n",
    "# print(allText[0])\n",
    "# print(allText[1])\n",
    "# print(allText[2])\n",
    "# print(allText[3])\n",
    "\n",
    "\n",
    "\n",
    "fileSegWordDonePath = r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed\\fileseg1.txt'\n",
    "with open(fileSegWordDonePath,'w',encoding='utf-8') as fW:\n",
    "    for text in allText:\n",
    "        fileTrainSeg = []\n",
    "        text = str(text)\n",
    "        if text is not None and text!='[]' :\n",
    "            sentence=jieba.lcut(text)\n",
    "            for word in sentence:\n",
    "                fileTrainSeg.append(word)\n",
    "            fW.write(\" \".join(fileTrainSeg))\n",
    "            fW.write('\\n')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "###################################################################################################\n",
    "\n",
    "\"\"\"\n",
    "gensim word2vec获取词向量\n",
    "\"\"\"\n",
    "\n",
    "import warnings\n",
    "import logging\n",
    "import os.path\n",
    "import sys\n",
    "import multiprocessing\n",
    "\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models.word2vec import LineSentence\n",
    "from gensim.models.word2vec import PathLineSentences\n",
    "\n",
    "# 忽略警告\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')\n",
    "\n",
    "\n",
    "program = os.path.basename(sys.argv[0]) # 读取当前文件的文件名\n",
    "logger = logging.getLogger(program)\n",
    "logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',level=logging.INFO)\n",
    "logger.info(\"running %s\" % ' '.join(sys.argv))\n",
    "\n",
    "inp = r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed\\fileseg1.txt'\n",
    "out_model = r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed\\corpusSegDone_1.model'\n",
    "out_vector = r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed\\corpusSegDone_1.vector' \n",
    "model = Word2Vec(LineSentence(inp), size=50, window=5, min_count=5, workers=multiprocessing.cpu_count())\n",
    "model.save(out_model)\n",
    "model.wv.save_word2vec_format(out_vector, binary=False)\n",
    "###########################\n",
    "\n",
    "\n",
    "\n",
    "##############词的频率\n",
    "\n",
    "\n",
    "import jieba\n",
    "file_path = r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed\\fileseg1.txt'\n",
    "# context = ''\n",
    "# with open(file_path, 'r', encoding='utf-8') as f:\n",
    "#         try:\n",
    "#             while True:\n",
    "#                 line = f.readline()\n",
    "#                 if line:\n",
    "#                     r = json.loads(line)\n",
    "#                     context = context +r\n",
    "#                 else:\n",
    "#                     break\n",
    "#         except:\n",
    "#             f.close()\n",
    "# print(context)\n",
    "\n",
    "# import jieba\n",
    "txt = open(file_path, \"r\", encoding='utf-8').read() \n",
    "# print(txt)\n",
    "words=jieba.lcut(txt)   \n",
    "counts = {}     # 通过键值对的形式存储词语及其出现的次数\n",
    "\n",
    "for word in words:\n",
    "    if  len(word) == 1:    # 单个词语不计算在内\n",
    "        continue\n",
    "    else:\n",
    "        counts[word] = counts.get(word, 0) + 1    # 遍历所有词语，每出现一次其对应的值加 1\n",
    "        \n",
    "items = list(counts.items())#将键值对转换成列表\n",
    "# print(items[0:3])\n",
    "# def quick_sort(qlist):\n",
    "#     if qlist == []:\n",
    "#         return []\n",
    "#     else:\n",
    "#         qfirst = qlist[0]\n",
    "#         qless = quick_sort([l for l in qlist[1:] if l[1] < qfirst[1]])\n",
    "#         qmore = quick_sort([m for m in qlist[1:] if m[1] >= qfirst[1]])\n",
    "#         return qmore + [qfirst] + qless\n",
    "\n",
    "items.sort(key=lambda x: x[1], reverse=True)    # 根据词语出现的次数进行从大到小排序\n",
    "items1=quick_sort(items)\n",
    "print(quick_sort(items))\n",
    "print('1')\n",
    "with open(r'C:\\Users\\Administrator.DESKTOP-BN41LK7\\Desktop\\preprocessed\\cipin.txt',mode='w',encoding='utf-8') as f2:\n",
    "    try:\n",
    "        while True:\n",
    "            for i in range(len(items1)):\n",
    "                word, count = items1[i]\n",
    "                print(\"{0:<5}{1:>5}\".format(word, count))\n",
    "                f2.write(str(word)+\" \"+str(count))\n",
    "                f2.write('\\n')\n",
    "    except:\n",
    "        f2.close()\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "['[', ']']\n[, ]\nFalse\n"
    }
   ],
   "source": [
    "##测试\n",
    "import jieba\n",
    "seg_list = jieba.lcut(\"[]\")  # 默认是精确模式\n",
    "seg_list += \"\"\n",
    "print(seg_list)\n",
    "\n",
    "print(\", \".join(seg_list))\n",
    "print(type(str(seg_list))==list) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[('from', 251), ('Who', 91), ('fled', 20)]\n[('from', 251), ('Who', 91), ('fled', 20)]\n"
    }
   ],
   "source": [
    "##测试\n",
    "\n",
    "list1 = [('Who', 91), ('fled', 20), ('from', 251)]\n",
    "def quick_sort(qlist):\n",
    "    if qlist == []:\n",
    "        return []\n",
    "    else:\n",
    "        qfirst = qlist[0]\n",
    "        qless = quick_sort([l for l in qlist[1:] if l[1] < qfirst[1]])\n",
    "        qmore = quick_sort([m for m in qlist[1:] if m[1] >= qfirst[1]])\n",
    "        return qmore + [qfirst] + qless\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "# list1.sort(key=lambda x: x[1],reverse=True)\n",
    "items=quick_sort(list1)\n",
    "print(items[0:3])\n",
    "print(quick_sort(list1))\n",
    "# print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[2, 3, 4, 5, 6, 6, 7, 8, 9]\n"
    }
   ],
   "source": [
    "##测试\n",
    "def quick_sort(qlist):\n",
    "    if qlist == []:\n",
    "        return []\n",
    "    else:\n",
    "        qfirst = qlist[0]\n",
    "        qless = quick_sort([l for l in qlist[1:] if l < qfirst])\n",
    "        qmore = quick_sort([m for m in qlist[1:] if m >= qfirst])\n",
    "        return qless + [qfirst] + qmore\n",
    " \n",
    "qlist = quick_sort([4,5,6,7,3,2,6,9,8])\n",
    "print(qlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}